package cn.dmp.tools

import cn.dmp.util.JedisPools
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import redis.clients.jedis.Jedis

/**
  * Created by Administrator on 2018/4/25.
  */
object AppDic2Redis {

  def main(args: Array[String]): Unit = {
    //判断路径参数
    if (args.length!=1){
      println(
        """
          |inputDataPath
          |参数路径不合法
        """.stripMargin)
      sys.exit()
    }
    //创建sparkContext
    val conf = new SparkConf()
      .setMaster("local[*]").setAppName("AppDic2Redis")
      .set("spark.serializer","org.apache.spark.serializer.KryoSerializer")
    val sc = new SparkContext(conf)

    //输入 输出的路径参数
    val Array(inputDataPath)=args

    val dicData = sc.textFile(inputDataPath)

    //处理原始数据
    val map= dicData.map(_.split("\t",-1)).filter(_.length>=5).map(t=>(t(4),t(1)))

    //写进redis
    map.foreachPartition(ite=> {
      val jedis: Jedis = JedisPools.getJedis

      ite.foreach(f=>{
          (jedis.hset("Appdic",f._1,f._2))
        })
      jedis.close()
    })
    sc.stop()
  }
}
